import pandas as pd
import multiprocessing
import os


# 多进程数据清洗
def process_excel(filename: list, in_folder_name: str, out_folder_name: str) -> None:
    for name in filename:
        print(in_folder_name + name)
        data = pd.read_csv(in_folder_name + name)
        # 将全是空值的列删除
        data.dropna(axis=1, how='all', inplace=True)
        # 用众数替换列空值
        data = data.T
        for i in range(data.shape[1]):
            x = data.iloc[3:, i].median()

            data.iloc[3:, i].fillna(x, inplace=True)
        data.T.to_csv("{0}/{1}".format(out_folder_name, name), encoding="gbk")


# 城市数据所在文件
city = "data/城市_20210101-20211231/"

# 站点数据所在文件
site = "data/站点_20210101-20211231/"


def get_process_num(filename_list: list, process_num: int = 4, in_folder_name: str = "data/城市_20210101-20211231/"):
    # 获取文件名列表
    filename = os.listdir(in_folder_name)
    process_list = []
    for i in range(0, len(filename_list), len(filename_list) // process_num):
        process_list.append(
            multiprocessing.Process(target=process_excel, kwargs={"out_folder_name": "outputs/city",
                                                                  "in_folder_name": in_folder_name,
                                                                  "filename": filename[i: i + process_num]})
        )
    return process_list


if __name__ == '__main__':
    # 城市数据处理进程
    process1 = get_process_num(os.listdir(city), in_folder_name=city)
    process2 = get_process_num(os.listdir(site), in_folder_name=site)
    for i in process1:
        i.start()

    for i in process2:
        i.start()
